package com.gxljc.bear.crawler.itaogao.cnstock;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.gxljc.commons.mongo.cli.BaseMongoRepository;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.NewsPageTemplate;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.MD5Util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;

import java.io.Serializable;
import java.util.*;

/**
 * 上海证券报 seed 数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-9-19
 */
public class CnstockSeedCrawler implements Serializable {
    public static Logger LOG = Logger.getLogger(CnstockSeedCrawler.class);
    private Boolean proxy = false;

    public CnstockSeedCrawler(Boolean proxy) {
        this.proxy = proxy;
    }

    public void crawl() throws Exception {
        List<String> seeds = getSeed();
        if (CollectionUtils.isEmpty(seeds)) {
            LOG.error("page is empty");
            return;
        }
        crawlNoSpark(seeds);
    }

    public void crawlNoSpark(List<String> seeds) {
        for (String url : seeds) {
            try {
                crawl(url);
            } catch (Exception e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }

    //单条url入口
    public int crawl(String url) throws Exception {
        LOG.info("crawl url = " + url);
        DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear.getName());
        String result = fetchData.getResult(url, proxy, ItaogaoConst.CRAWL_TIMEOUT);
        if (StringUtils.isEmpty(result)) {
            LOG.error("page result is empty url = " + url);
            return DolphinCrawlerConsts.RESULT_NO;
        }
        List<NewsPageTable> tables = extract(result, url);
        MongodbUtil.saveMongodb(tables, ItaogaoConst.MONGODB_TABLE.CNSTOCK.getValue());
        return DolphinCrawlerConsts.RESULT_YES;
    }

    private List<NewsPageTable> extract(String result, String url) throws Exception {
        final Gson GSON = new Gson();
        NewsPageTemplate template = GSON.fromJson(result, new TypeToken<NewsPageTemplate>() {
        }.getType());
        if (template == null) return null;
        return parseObject(template, url);
    }

    private List<NewsPageTable> parseObject(NewsPageTemplate template, String url) throws Exception {
        List<NewsPageTable> tables = new LinkedList<>();
        List<String> crawlUrls = template.getCrawlUrl();
        if (CollectionUtils.isEmpty(crawlUrls)) return null;
        int size = crawlUrls.size();
        for (int i = 0; i < size; i++) {
            String crawlUrl = crawlUrls.get(i);
            NewsPageTable table = new NewsPageTable();
            crawlUrl = HtmlUtil.pictureUrlPref(crawlUrl, url);
            table.setId(genId(crawlUrl));
            table.setCrawlFlag(0);
            table.setCrawlUrl(crawlUrl);
            tables.add(table);
        }
        return tables;
    }

    //生成id
    private String genId(String href) {
        return MD5Util.getMd5(href);
    }

    //构造URL seed
    public List<String> getSeed() throws Exception {
        Map<String, Integer> seeds = new HashMap<String, Integer>();
        seeds.put("http://news.cnstock.com/news/sns_yw/%s", 18);
        seeds.put("http://company.cnstock.com/company/scp_gsxw/%s", 115);
        seeds.put("http://ggjd.cnstock.com/gglist/search/qmtbbdj/%s", 5);
        seeds.put("http://company.cnstock.com/company/scp_dsy/tcsy_rdgs/%s", 14);
        List<String> urls = new LinkedList<String>();
        for (Map.Entry seed : seeds.entrySet()) {
            int max = (Integer) seed.getValue();
            String urlFormat = (String) seed.getKey();
            for (int i = 1; i <= max; i++) {
                String url = String.format(urlFormat, i + "");
                urls.add(url);
            }
        }
        return urls;
    }

}
